import numpy as np
import pandas as pd
import xgboost as xgb
import lightgbm as lgb
import gc
import random
import datetime as dt
from datetime import datetime
import xgbmodel
import lgbmodel
import rfmodel
import gbmmodel
import adamodel 
import stack
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error as mae

# Parameters
# XGB1_WEIGHT = 0.1561
BASELINE_WEIGHT = 0.0024#0.0039#
OLS_WEIGHT = 0.04
XGB0_WEIGHT = 0.43#0.63#
RF_WEIGHT=0.04#0.00063#
GBM_WEIGHT=0.41
#LGB_WEIGHT=0.240#
# ADA_WEIGHT=0.05


# OLS_WEIGHT = 0.04
# XGB0_WEIGHT=0.35
#STACK_WEIGHT = 0.09
STACK_WEIGHT = 0.1

BASELINE_PRED = 0.0115   # Baseline based on mean of training data, per Oleg

np.random.seed(0)
random.seed(0)

#从磁盘读取时间预测结果
print('读时间预测结果')
xgb_pred0 = pd.read_csv(r'.\input\date_predict.csv')
#预测生成时间预测结果
# print('重新生成时间预测结果')
# xgb_pred0 = xgbmodel.xgb0_train()()
#print(xgb_pred0.head())

#XGB 模型预测
print('读xgb1预测结果')
df_xgb_pred1 = pd.read_csv(r'.\input\xgb1_predict.csv')
xgb_pred1=df_xgb_pred1['predict'].values.astype(np.float32)
# xgb_pred1 = xgbmodel.xgb1_train()
#LGB 模型预测
print('读lgb预测结果')
df_lgb_pred = pd.read_csv(r'.\input\lgb_predict.csv')
lgb_pred = df_lgb_pred['predict'].values.astype(np.float32)
# lgb_pred = lgbmodel.lgb_train()
#RF 模型预测
print('读rf预测结果')
df_rf_pred = pd.read_csv(r'.\input\rf_predict.csv')
rf_pred=df_rf_pred['predict'].values.astype(np.float32)
rf_pred=rf_pred*1.4
#rf_pred = rfmodel.rf_train()
#GBM 模型预测
print('读gbm预测结果')
df_gbm_predict = pd.read_csv(r'.\input\gbm_predict.csv')
gbm_pred=df_gbm_predict['predict'].values.astype(np.float32)
# gbm_pred = gbmmodel.gbm_train()
#ADA 模型预测
print('读ada预测结果')
df_ada_predict = pd.read_csv(r'.\input\ada_predict.csv')
ada_pred=df_ada_predict['predict'].values.astype(np.float32)
#ada_pred=adamodel.ada_train()

#stack 模型预测
print('读stack预测结果')
df_stack_pred = pd.read_csv(r'.\input\stack_predict.csv')
stack_pred=df_stack_pred['predict'].values.astype(np.float32)
# stack_pred=stack.stacking()


np.random.seed(17)
random.seed(17)

train = pd.read_csv(r".\input\train_2016_v2.csv",parse_dates=["transactiondate"])
properties = pd.read_csv(r".\input\properties_2016.csv")
submission = pd.read_csv(r'.\input\sample_submission.csv')
properties['propertycountylandusecode'].fillna(
    properties['propertycountylandusecode'].mode()[0])
properties = properties.drop(['regionidcity'], axis=1)


def get_features(df):
    df["transactiondate"] = pd.to_datetime(df["transactiondate"])
    # df["transactiondate_year"] = df["transactiondate"].dt.year
    df["transactiondate"] = df["transactiondate"].dt.month.astype(np.int64)
    # df['transactiondate'] = df['transactiondate'].dt.quarter
    df = df.fillna(-1.0)
    return df


def MAE(y, ypred):
    # logerror=log(Zestimate)−log(SalePrice)
    return np.sum([abs(y[i] - ypred[i]) for i in range(len(y))])


train = pd.merge(train, properties, how='left', on='parcelid')
y = train['logerror'].values
test = pd.merge(submission, properties, how='left',
                left_on='ParcelId', right_on='parcelid')
properties = []  # memory

exc = [train.columns[c] for c in range(
    len(train.columns)) if train.dtypes[c] == 'O'] + ['logerror', 'parcelid']
col = [c for c in train.columns if c not in exc]
col = ['fullbathcnt','calculatedfinishedsquarefeet','transactiondate']
train = get_features(train[col])
# should use the most common training date
test['transactiondate'] = '2016-01-01'
test = get_features(test[col])

reg = LinearRegression(n_jobs=-1)
reg.fit(train, y)
print('fit...')
print(mae(y, reg.predict(train)))
train = []
y = []  # memory

LGB_WEIGHT=(1 - XGB0_WEIGHT-BASELINE_WEIGHT-GBM_WEIGHT)
print('LGB_WEIGHT:%0.5f'%(LGB_WEIGHT))
pred1 =  LGB_WEIGHT*lgb_pred \
		+ BASELINE_WEIGHT*BASELINE_PRED \
		+ GBM_WEIGHT*gbm_pred  
test_dates = ['2016-10-01', '2016-11-01', '2016-12-01',
              '2017-10-01', '2017-11-01', '2017-12-01']
test_columns = ['201610', '201611', '201612', '201710', '201711', '201712']
for i in range(len(test_dates)):
    test['transactiondate'] = test_dates[i]
    pred2=XGB0_WEIGHT * xgb_pred0[test_columns[i]].values.astype(np.float32) + pred1
    pred3=STACK_WEIGHT * stack_pred + (1-STACK_WEIGHT) * pred2
    pred = OLS_WEIGHT*reg.predict(get_features(test))  + pred3
    pred = pred*1.05
    submission[test_columns[i]] = [float(format(x, '.4f')) for x in pred]

# 写结果到磁盘
print('\n写结果到磁盘中..')
submission.to_csv(r'.\result\sub{}.csv'.format(
    datetime.now().strftime('%Y%m%d_%H%M%S')), index=False)
print('结束')
#29
#恢复原有模型 stack模型 预测结果乘以系数1.05 LB 0.0642586
#

#28
#STACK 融合
#STACK 0.1 BASELINE_WEIGHT=0.0023  pred预测乘以系数1.1  LB 0.0642664

#27
#RF融合 系数进入初始层
#BASELINE_WEIGHT=0.0023 XGB0_WEIGHT=0.41 RF_WEIGHT=0.07 GBM_WEIGHT=0.39 LGB_WEIGHT=0.12770   LB 0.0642678
#GBM_WEIGHT=0.4  XGB0_WEIGHT=0.42 LGB_WEIGHT=0.10770 LB 0.0642687
#GBM_WEIGHT=0.37 XGB0_WEIGHT=0.42 LGB_WEIGHT=0.13770 LB 0.0642688
#GBM_WEIGHT=0.39 XGB0_WEIGHT=0.41 LGB_WEIGHT=0.09770 RF_WEIGHT=0.1  LB 0.0642719
#GBM_WEIGHT=0.39 XGB0_WEIGHT=0.41 LGB_WEIGHT=0.13770 RF_WEIGHT=0.06 LB 0.0642663  
#GBM_WEIGHT=0.39 XGB0_WEIGHT=0.40 LGB_WEIGHT=0.14770 RF_WEIGHT=0.06 LB 0.0642668
#GBM_WEIGHT=0.39 XGB0_WEIGHT=0.40 LGB_WEIGHT=0.12770 RF_WEIGHT=0.08 LB 0.0642690
#GBM_WEIGHT=0.39 XGB0_WEIGHT=0.41 LGB_WEIGHT=0.15770 RF_WEIGHT=0.04 LB 0.0642645

#26
#修改RF系数后，与整体融合
#RF_WEIGHT 0.1  LB 0.0642717
#RF_WEIGHT 0.05 LB 0.0642657

#25
#单个RF 乘以系数
#系数 1.0 LB 0.0647977
#系数 1.2 LB 0.0646539
#系数 1.3 LB 0.0646299

#24
#stack 原有模型
#STACK_WEIGHT 0.1  LB 0.0642622
#STACK_WEIGHT 0.09 LB 0.0642623

#23 
#原有模型乘以系数1.05 LB 0.0642557
#原有模型乘以系数1.1  LB 0.0642598

#22
#stack 9个模型 单独 
#棵数=350 sub_feature=0.5 max_depth=5 LB 0.0644017 
#棵数=400 sub_feature=1   max_depth=6 LB 0.0644282
#21
#stack
#STACK_WEIGHT=0.05   LB 
#STACK_WEIGHT=0.07   LB 0.0642587
#STACK_WEIGHT=0.1    LB 0.0642579
#STACK_WEIGHT=0.11   LB 0.0642585
#STACK_WEIGHT=0.2    LB 
#20
#GBM 采样0.7
#LGB_WEIGHT=0.15760  GBM_WEIGHT=0.41  XGB0_WEIGHT=0.43 LB 0.0642763
#GBM 3倍预测时间
#LGB_WEIGHT=0.15760  GBM_WEIGHT=0.41  XGB0_WEIGHT=0.43 LB 0.0642880

#19
#新权重
#BASELINE_WEIGHT=0.0024 OLS_WEIGHT=0.04 XGB0_WEIGHT=0.4 LGB_WEIGHT=0.15 RF_WEIGHT=0.00063 GBM_WEIGHT=0.37 LB 0.0642676
#LGB_WEIGHT=0.13 xgb1_weight=0.09760 LB 0.0642709
#LGB_WEIGHT=0.15 xgb1_weight=0.07760 GBM_WEIGHT=0.35  XGB0_WEIGHT=0.42 LB 0.0642697
#LGB_WEIGHT=0.15 xgb1_weight=0.02760 GBM_WEIGHT=0.37  XGB0_WEIGHT=0.45 LB 0.0642675
#LGB_WEIGHT=0.15 xgb1_weight=0.00060 GBM_WEIGHT=0.397 XGB0_WEIGHT=0.45 LB 0.0642651
#LGB_WEIGHT=0.15760  xgb1_weight=0.0  GBM_WEIGHT=0.4   XGB0_WEIGHT=0.44 LB 0.0642644
#LGB_WEIGHT=0.15760  GBM_WEIGHT=0.41  XGB0_WEIGHT=0.43 LB 0.0642639
#LGB_WEIGHT=0.13760  GBM_WEIGHT=0.43  XGB0_WEIGHT=0.43 LB 0.0642648
#LGB_WEIGHT=0.15760  GBM_WEIGHT=0.41  XGB0_WEIGHT=0.43 LB 

#18 
#RF_WEIGHT 0.001 XGB0 num_rounds=260 LB 0.0642689
#XGB0 num_rounds=250 max_dep=5 LB 0.0642998 
#XGB0 num_rounds=260 max_dep=5 LB 0.0642979
#XGB0 num_rounds=260 max_dep=6 XGB0 max_dep=6 LB 0.0642688
#XGB0_WEIGHT=0.63 LB 0.0642678

#17 
#LGB_WEIGHT=0.240 ADA_WEIGHT=0 RF RF_WEIGHT 0.0035 额外添加 LB 0.0642996
#LGB_WEIGHT=0.240 ADA_WEIGHT=0 RF RF_WEIGHT 0.005  额外添加 LB 0.0642999

#16
#LGB_WEIGHT=0.240 ADA_WEIGHT=0.07 LB 0.0643070
#LGB_WEIGHT=0.240 ADA_WEIGHT=0.05 LB 0.0643052
#LGB_WEIGHT=0.200 ADA_WEIGHT=0.05 LB 0.0643054

#15
#加入ADA模型
#ADA_WEIGHT 0.1 	LB 0.0643224
#ADA_WEIGHT 0.02 	LB 0.0643034
#放入里面 ADA_WEIGHT 0.05 	LB 0.0642976
#放入里面 ADA_WEIGHT 0.07 	LB 0.0642973
#LGB_WEIGHT 0.22  ADA_WEIGHT 0.07  	LB 0.0642979
#LGB_WEIGHT=0.22  ADA_WEIGHT=0.09  乘以系数1.001	LB 0.0642985

#14
#使用RF替换BASELINE_PRED
#RF_WEIGHT 0.003	LB 0.0642997
#RF_WEIGHT 0.004	LB 0.0642999
#RF_WEIGHT 0.0035	gbm去除特征工程  LB 0.0643107
#OLS_WEIGHT 0.035   LB 0.0643022
#13
# BASELINE_WEIGHT=0.0039 LGB_WEIGHT=0.200     LB 0.0643070
# BASELINE_WEIGHT=0.0039 LGB_WEIGHT=0.220     LB 0.0643054
# LGB_WEIGHT=0.240                            LB 0.0643045
# XGB0_WEIGHT=0.5   GBM_WEIGHT=0.37           LB 0.0643037
# XGB0_WEIGHT=0.55                            LB 0.0643016
# XGB0_WEIGHT=0.6                             LB 0.0643001

#12
#单个GBM n_estimators 300, max_depth 7, learning_rate 0.05,subsample 0.8,max_features 0.35 LB 0.0646073
#GBM GBM_WEIGHT 0.1         LB 0.0643494
#GBM GBM_WEIGHT 0.15        LB 0.0643338 添加特征值
#GBM GBM_WEIGHT 0.2         LB 0.0643232
#GBM GBM_WEIGHT 0.3         LB 0.0643113
#GBM GBM_WEIGHT 0.35        LB 0.0643098
#GBM GBM_WEIGHT 0.4         LB 0.0643110
#GBM max_depth=5            LB 0.0643254
#GBM max_features 0.36      LB 0.0643055
#GBM max_features 0.37      LB 0.0643055

#11 
# OLS 0.04 加入parcelid 
# XGB1 采样0.75 棵数220 eta0.033  LB 0.0643963
# XGB1 采样0.75 棵数260 eta0.033  BASELINE_PRED 0.0056 LB 0.0643923
# XGB1 采样0.75 棵数280 eta0.033  LB 0.0643939
# XGB1 采样0.75 棵数300 eta0.035  LB 0.0643984
# XGB1 采样0.8  棵数260 eta0.035  LB 0.0643962
# XGB1 采样0.8  棵数250 eta0.037  BASELINE_PRED 0.0034 LB 0.0643869

#10
#OLS使用fullbathcnt, calculatedfinishedsquarefeet,transactiondate三个特征
#OLS 0.029 LB 0.0643878
#OLS 0.033 LB 0.0643873
#OLS 0.05  LB 0.0643875

#9
#XGB1 gamma=0   XGB0 num_rounds=300   LB  0.0643906

#8 
#XGB1 eta 0.03  LB 0.0643988
#XGB1 eta 0.03  num_rounds=150                                 LB 0.0643971
#XGB1 eta 0.03  num_rounds=150  XGB0_WEIGHT 0.5                LB 0.0643985
#XGB1 eta:0.03  num_rounds=300  max_depth=6 XGB0_WEIGHT 0.47   LB 0.0643978
#XGB1 eta:0.03  num_rounds=300  max_depth=5   LGB max_bin=20   LB 0.0643986
#XGB1 eta:0.037 num_rounds=250  gamma=0.001   LGB max_bin=10   LB 0.0643942

#7 
#OLS 0.3    LGB 特征采样系数0.35  LB 0.0646564
#OLS 0.029  LGB 特征采样系数0.35  LB 0.0644171
#OLS 0.029  LGB 特征采样系数0.345 LB 0.0644169
#OLS 0.029  LGB 特征采样系数0.36  LB 0.0643893
#OLS 0.029  LGB 特征采样系数0.365 LB 0.0643893
#OLS 0.029  LGB 特征采样系数0.375 LB 0.06439
#OLS 0.029  LGB 特征采样系数0.39  LB 0.0643913

#6 
#修改XGB0 取消季度特征, 
#OLS 0.034 LB 0.0644180
#OLS 0.029 LB 0.0644169
#OLS 0.026 LB 0.0644173

#5 添加最小二乘法
# XGB0_WEIGHT 0.47  LGB_WEIGHT 0.22    xgb_weight 0.2234 
# 系数  0.04  LB 0.0644274
# 系数  0.047 LB 0.0644286
# 系数  0.055 LB 0.0644315
# 系数  0.07  LB 0.0644363

#4 
# XGB0_WEIGHT 0.47  LGB_WEIGHT 0.2290   xgb_weight 0.2144   BASELINE_WEIGHT 0.0056   LB 0.0644383
# XGB0_WEIGHT 0.47  LGB_WEIGHT 0.22     xgb_weight 0.2234   BASELINE_WEIGHT 0.0056   LB 0.0644384

#3
#单个LGB LB 0.0647120
#单个LGB LB 增加异常值去除 0.0647504

#2 
# XGB0_WEIGHT 0.45  LGB_WEIGHT 0.40   xgb_weight 0.15    LB 0.0644553
# XGB0_WEIGHT 0.45  LGB_WEIGHT 0.33   xgb_weight 0.22    LB 0.0644445
# XGB0_WEIGHT 0.45  LGB_WEIGHT 0.26   xgb_weight 0.29    LB 0.0644376
# XGB0_WEIGHT 0.55  LGB_WEIGHT 0.2290 xgb_weight 0.221   LB 0.0644474
# XGB0_WEIGHT 0.45  LGB_WEIGHT 0.22   xgb_weight 0.33    LB 0.0644369
# XGB0_WEIGHT 0.45  LGB_WEIGHT 0.18   xgb_weight 0.37    LB 0.0644395

#1
# 单个XGB XGB时间，提高时间占比到0.80
# XGB0_WEIGHT 0.8    LB 0.0644898
# XGB0_WEIGHT 0.55   LB 0.0644684
# XGB0_WEIGHT 0.5    LB 0.0644647
# XGB0_WEIGHT 0.45   LB 0.0644658
# XGB0_WEIGHT 0.3    LB 0.0644770
# XGB0_WEIGHT 0.08   LB 0.0645159
# 暂时将 XGB0_WEIGHT 设为0.45


